

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta name="Description" content="scikit-learn: machine learning in Python">

  
  <title>Working With Text Data &mdash; scikit-learn 0.22 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html" />

  
  <link rel="shortcut icon" href="../../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
<script src="../../_static/jquery.js"></script> 
</head>
<body>
<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../../index.html">
        <img
          class="sk-brand-img"
          src="../../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../modules/classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../getting_started.html">Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../index.html">Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../glossary.html">Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../developers/index.html">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../faq.html">FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../related_projects.html">Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../roadmap.html">Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../about.html">About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../../getting_started.html">Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../index.html">Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../glossary.html">Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../developers/index.html">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../faq.html">FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../related_projects.html">Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../roadmap.html">Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../about.html">About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="sk-sidebar-toc-logo">
          <a href="../../index.html">
            <img
              class="sk-brand-img"
              src="../../_static/scikit-learn-logo-small.png"
              alt="logo"/>
          </a>
        </div>
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="../statistical_inference/finding_help.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="Finding help">Prev</a><a href="../index.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="scikit-learn Tutorials">Up</a>
            <a href="../machine_learning_map/index.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="Choosing the right estimator">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 0.22</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
          <div class="sk-sidebar-toc">
            <ul>
<li><a class="reference internal" href="#">Working With Text Data</a><ul>
<li><a class="reference internal" href="#tutorial-setup">Tutorial setup</a></li>
<li><a class="reference internal" href="#loading-the-20-newsgroups-dataset">Loading the 20 newsgroups dataset</a></li>
<li><a class="reference internal" href="#extracting-features-from-text-files">Extracting features from text files</a><ul>
<li><a class="reference internal" href="#bags-of-words">Bags of words</a></li>
<li><a class="reference internal" href="#tokenizing-text-with-scikit-learn">Tokenizing text with <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code></a></li>
<li><a class="reference internal" href="#from-occurrences-to-frequencies">From occurrences to frequencies</a></li>
</ul>
</li>
<li><a class="reference internal" href="#training-a-classifier">Training a classifier</a></li>
<li><a class="reference internal" href="#building-a-pipeline">Building a pipeline</a></li>
<li><a class="reference internal" href="#evaluation-of-the-performance-on-the-test-set">Evaluation of the performance on the test set</a></li>
<li><a class="reference internal" href="#parameter-tuning-using-grid-search">Parameter tuning using grid search</a><ul>
<li><a class="reference internal" href="#exercises">Exercises</a></li>
</ul>
</li>
<li><a class="reference internal" href="#exercise-1-language-identification">Exercise 1: Language identification</a></li>
<li><a class="reference internal" href="#exercise-2-sentiment-analysis-on-movie-reviews">Exercise 2: Sentiment Analysis on movie reviews</a></li>
<li><a class="reference internal" href="#exercise-3-cli-text-classification-utility">Exercise 3: CLI text classification utility</a></li>
<li><a class="reference internal" href="#where-to-from-here">Where to from here</a></li>
</ul>
</li>
</ul>

          </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <div class="section" id="working-with-text-data">
<span id="text-data-tutorial"></span><h1>Working With Text Data<a class="headerlink" href="#working-with-text-data" title="Permalink to this headline">¶</a></h1>
<p>The goal of this guide is to explore some of the main <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code>
tools on a single practical task: analyzing a collection of text
documents (newsgroups posts) on twenty different topics.</p>
<p>In this section we will see how to:</p>
<blockquote>
<div><ul class="simple">
<li><p>load the file contents and the categories</p></li>
<li><p>extract feature vectors suitable for machine learning</p></li>
<li><p>train a linear model to perform categorization</p></li>
<li><p>use a grid search strategy to find a good configuration of both
the feature extraction components and the classifier</p></li>
</ul>
</div></blockquote>
<div class="section" id="tutorial-setup">
<h2>Tutorial setup<a class="headerlink" href="#tutorial-setup" title="Permalink to this headline">¶</a></h2>
<p>To get started with this tutorial, you must first install
<em>scikit-learn</em> and all of its required dependencies.</p>
<p>Please refer to the <a class="reference internal" href="../../install.html#installation-instructions"><span class="std std-ref">installation instructions</span></a>
page for more information and for system-specific instructions.</p>
<p>The source of this tutorial can be found within your scikit-learn folder:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">scikit</span><span class="o">-</span><span class="n">learn</span><span class="o">/</span><span class="n">doc</span><span class="o">/</span><span class="n">tutorial</span><span class="o">/</span><span class="n">text_analytics</span><span class="o">/</span>
</pre></div>
</div>
<p>The source can also be found <a class="reference external" href="https://github.com/scikit-learn/scikit-learn/tree/master/doc/tutorial/text_analytics">on Github</a>.</p>
<p>The tutorial folder should contain the following sub-folders:</p>
<blockquote>
<div><ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">*.rst</span> <span class="pre">files</span></code> - the source of the tutorial document written with sphinx</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">data</span></code> - folder to put the datasets used during the tutorial</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">skeletons</span></code> - sample incomplete scripts for the exercises</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">solutions</span></code> - solutions of the exercises</p></li>
</ul>
</div></blockquote>
<p>You can already copy the skeletons into a new folder somewhere
on your hard-drive named <code class="docutils literal notranslate"><span class="pre">sklearn_tut_workspace</span></code> where you
will edit your own files for the exercises while keeping
the original skeletons intact:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">%</span> <span class="n">cp</span> <span class="o">-</span><span class="n">r</span> <span class="n">skeletons</span> <span class="n">work_directory</span><span class="o">/</span><span class="n">sklearn_tut_workspace</span>
</pre></div>
</div>
<p>Machine learning algorithms need data. Go to each <code class="docutils literal notranslate"><span class="pre">$TUTORIAL_HOME/data</span></code>
sub-folder and run the <code class="docutils literal notranslate"><span class="pre">fetch_data.py</span></code> script from there (after
having read them first).</p>
<p>For instance:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>% cd $TUTORIAL_HOME/data/languages
% less fetch_data.py
% python fetch_data.py
</pre></div>
</div>
</div>
<div class="section" id="loading-the-20-newsgroups-dataset">
<h2>Loading the 20 newsgroups dataset<a class="headerlink" href="#loading-the-20-newsgroups-dataset" title="Permalink to this headline">¶</a></h2>
<p>The dataset is called “Twenty Newsgroups”. Here is the official
description, quoted from the <a class="reference external" href="http://people.csail.mit.edu/jrennie/20Newsgroups/">website</a>:</p>
<blockquote>
<div><p>The 20 Newsgroups data set is a collection of approximately 20,000
newsgroup documents, partitioned (nearly) evenly across 20 different
newsgroups. To the best of our knowledge, it was originally collected
by Ken Lang, probably for his paper “Newsweeder: Learning to filter
netnews,” though he does not explicitly mention this collection.
The 20 newsgroups collection has become a popular data set for
experiments in text applications of machine learning techniques,
such as text classification and text clustering.</p>
</div></blockquote>
<p>In the following we will use the built-in dataset loader for 20 newsgroups
from scikit-learn. Alternatively, it is possible to download the dataset
manually from the website and use the <a class="reference internal" href="../../modules/generated/sklearn.datasets.load_files.html#sklearn.datasets.load_files" title="sklearn.datasets.load_files"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.load_files</span></code></a>
function by pointing it to the <code class="docutils literal notranslate"><span class="pre">20news-bydate-train</span></code> sub-folder of the
uncompressed archive folder.</p>
<p>In order to get faster execution times for this first example we will
work on a partial dataset with only 4 categories out of the 20 available
in the dataset:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;alt.atheism&#39;</span><span class="p">,</span> <span class="s1">&#39;soc.religion.christian&#39;</span><span class="p">,</span>
<span class="gp">... </span>              <span class="s1">&#39;comp.graphics&#39;</span><span class="p">,</span> <span class="s1">&#39;sci.med&#39;</span><span class="p">]</span>
</pre></div>
</div>
<p>We can now load the list of files matching those categories as follows:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_20newsgroups</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">twenty_train</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)</span>
</pre></div>
</div>
<p>The returned dataset is a <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> “bunch”: a simple holder
object with fields that can be both accessed as python <code class="docutils literal notranslate"><span class="pre">dict</span></code>
keys or <code class="docutils literal notranslate"><span class="pre">object</span></code> attributes for convenience, for instance the
<code class="docutils literal notranslate"><span class="pre">target_names</span></code> holds the list of the requested category names:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target_names</span>
<span class="go">[&#39;alt.atheism&#39;, &#39;comp.graphics&#39;, &#39;sci.med&#39;, &#39;soc.religion.christian&#39;]</span>
</pre></div>
</div>
<p>The files themselves are loaded in memory in the <code class="docutils literal notranslate"><span class="pre">data</span></code> attribute. For
reference the filenames are also available:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">len</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="go">2257</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">len</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">filenames</span><span class="p">)</span>
<span class="go">2257</span>
</pre></div>
</div>
<p>Let’s print the first lines of the first loaded file:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)[:</span><span class="mi">3</span><span class="p">]))</span>
<span class="go">From: sd345@city.ac.uk (Michael Collier)</span>
<span class="go">Subject: Converting images to HP LaserJet III?</span>
<span class="go">Nntp-Posting-Host: hampton</span>

<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">[</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">[</span><span class="mi">0</span><span class="p">]])</span>
<span class="go">comp.graphics</span>
</pre></div>
</div>
<p>Supervised learning algorithms will require a category label for each
document in the training set. In this case the category is the name of the
newsgroup which also happens to be the name of the folder holding the
individual documents.</p>
<p>For speed and space efficiency reasons <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> loads the
target attribute as an array of integers that corresponds to the
index of the category name in the <code class="docutils literal notranslate"><span class="pre">target_names</span></code> list. The category
integer id of each sample is stored in the <code class="docutils literal notranslate"><span class="pre">target</span></code> attribute:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">10</span><span class="p">]</span>
<span class="go">array([1, 1, 3, 3, 3, 3, 3, 2, 2, 2])</span>
</pre></div>
</div>
<p>It is possible to get back the category names as follows:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">10</span><span class="p">]:</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">[</span><span class="n">t</span><span class="p">])</span>
<span class="gp">...</span>
<span class="go">comp.graphics</span>
<span class="go">comp.graphics</span>
<span class="go">soc.religion.christian</span>
<span class="go">soc.religion.christian</span>
<span class="go">soc.religion.christian</span>
<span class="go">soc.religion.christian</span>
<span class="go">soc.religion.christian</span>
<span class="go">sci.med</span>
<span class="go">sci.med</span>
<span class="go">sci.med</span>
</pre></div>
</div>
<p>You might have noticed that the samples were shuffled randomly when we called
<code class="docutils literal notranslate"><span class="pre">fetch_20newsgroups(...,</span> <span class="pre">shuffle=True,</span> <span class="pre">random_state=42)</span></code>: this is useful if
you wish to select only a subset of samples to quickly train a model and get a
first idea of the results before re-training on the complete dataset later.</p>
</div>
<div class="section" id="extracting-features-from-text-files">
<h2>Extracting features from text files<a class="headerlink" href="#extracting-features-from-text-files" title="Permalink to this headline">¶</a></h2>
<p>In order to perform machine learning on text documents, we first need to
turn the text content into numerical feature vectors.</p>
<div class="section" id="bags-of-words">
<h3>Bags of words<a class="headerlink" href="#bags-of-words" title="Permalink to this headline">¶</a></h3>
<p>The most intuitive way to do so is to use a bags of words representation:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p>Assign a fixed integer id to each word occurring in any document
of the training set (for instance by building a dictionary
from words to integer indices).</p></li>
<li><p>For each document <code class="docutils literal notranslate"><span class="pre">#i</span></code>, count the number of occurrences of each
word <code class="docutils literal notranslate"><span class="pre">w</span></code> and store it in <code class="docutils literal notranslate"><span class="pre">X[i,</span> <span class="pre">j]</span></code> as the value of feature
<code class="docutils literal notranslate"><span class="pre">#j</span></code> where <code class="docutils literal notranslate"><span class="pre">j</span></code> is the index of word <code class="docutils literal notranslate"><span class="pre">w</span></code> in the dictionary.</p></li>
</ol>
</div></blockquote>
<p>The bags of words representation implies that <code class="docutils literal notranslate"><span class="pre">n_features</span></code> is
the number of distinct words in the corpus: this number is typically
larger than 100,000.</p>
<p>If <code class="docutils literal notranslate"><span class="pre">n_samples</span> <span class="pre">==</span> <span class="pre">10000</span></code>, storing <code class="docutils literal notranslate"><span class="pre">X</span></code> as a NumPy array of type
float32 would require 10000 x 100000 x 4 bytes = <strong>4GB in RAM</strong> which
is barely manageable on today’s computers.</p>
<p>Fortunately, <strong>most values in X will be zeros</strong> since for a given
document less than a few thousand distinct words will be
used. For this reason we say that bags of words are typically
<strong>high-dimensional sparse datasets</strong>. We can save a lot of memory by
only storing the non-zero parts of the feature vectors in memory.</p>
<p><code class="docutils literal notranslate"><span class="pre">scipy.sparse</span></code> matrices are data structures that do exactly this,
and <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> has built-in support for these structures.</p>
</div>
<div class="section" id="tokenizing-text-with-scikit-learn">
<h3>Tokenizing text with <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code><a class="headerlink" href="#tokenizing-text-with-scikit-learn" title="Permalink to this headline">¶</a></h3>
<p>Text preprocessing, tokenizing and filtering of stopwords are all included
in <a class="reference internal" href="../../modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer" title="sklearn.feature_extraction.text.CountVectorizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">CountVectorizer</span></code></a>, which builds a dictionary of features and
transforms documents to feature vectors:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.feature_extraction.text</span> <span class="kn">import</span> <span class="n">CountVectorizer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">count_vect</span> <span class="o">=</span> <span class="n">CountVectorizer</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_counts</span> <span class="o">=</span> <span class="n">count_vect</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_counts</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2257, 35788)</span>
</pre></div>
</div>
<p><a class="reference internal" href="../../modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer" title="sklearn.feature_extraction.text.CountVectorizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">CountVectorizer</span></code></a> supports counts of N-grams of words or consecutive
characters. Once fitted, the vectorizer has built a dictionary of feature
indices:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">count_vect</span><span class="o">.</span><span class="n">vocabulary_</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="sa">u</span><span class="s1">&#39;algorithm&#39;</span><span class="p">)</span>
<span class="go">4690</span>
</pre></div>
</div>
<p>The index value of a word in the vocabulary is linked to its frequency
in the whole training corpus.</p>
</div>
<div class="section" id="from-occurrences-to-frequencies">
<h3>From occurrences to frequencies<a class="headerlink" href="#from-occurrences-to-frequencies" title="Permalink to this headline">¶</a></h3>
<p>Occurrence count is a good start but there is an issue: longer
documents will have higher average count values than shorter documents,
even though they might talk about the same topics.</p>
<p>To avoid these potential discrepancies it suffices to divide the
number of occurrences of each word in a document by the total number
of words in the document: these new features are called <code class="docutils literal notranslate"><span class="pre">tf</span></code> for Term
Frequencies.</p>
<p>Another refinement on top of tf is to downscale weights for words
that occur in many documents in the corpus and are therefore less
informative than those that occur only in a smaller portion of the
corpus.</p>
<p>This downscaling is called <a class="reference external" href="https://en.wikipedia.org/wiki/Tf-idf">tf–idf</a> for “Term Frequency times
Inverse Document Frequency”.</p>
<p>Both <strong>tf</strong> and <strong>tf–idf</strong> can be computed as follows using
<a class="reference internal" href="../../modules/generated/sklearn.feature_extraction.text.TfidfTransformer.html#sklearn.feature_extraction.text.TfidfTransformer" title="sklearn.feature_extraction.text.TfidfTransformer"><code class="xref py py-class docutils literal notranslate"><span class="pre">TfidfTransformer</span></code></a>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.feature_extraction.text</span> <span class="kn">import</span> <span class="n">TfidfTransformer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tf_transformer</span> <span class="o">=</span> <span class="n">TfidfTransformer</span><span class="p">(</span><span class="n">use_idf</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train_counts</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_tf</span> <span class="o">=</span> <span class="n">tf_transformer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_train_counts</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_tf</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2257, 35788)</span>
</pre></div>
</div>
<p>In the above example-code, we firstly use the <code class="docutils literal notranslate"><span class="pre">fit(..)</span></code> method to fit our
estimator to the data and secondly the <code class="docutils literal notranslate"><span class="pre">transform(..)</span></code> method to transform
our count-matrix to a tf-idf representation.
These two steps can be combined to achieve the same end result faster
by skipping redundant processing. This is done through using the
<code class="docutils literal notranslate"><span class="pre">fit_transform(..)</span></code> method as shown below, and as mentioned in the note
in the previous section:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">tfidf_transformer</span> <span class="o">=</span> <span class="n">TfidfTransformer</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_tfidf</span> <span class="o">=</span> <span class="n">tfidf_transformer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">X_train_counts</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train_tfidf</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2257, 35788)</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="training-a-classifier">
<h2>Training a classifier<a class="headerlink" href="#training-a-classifier" title="Permalink to this headline">¶</a></h2>
<p>Now that we have our features, we can train a classifier to try to predict
the category of a post. Let’s start with a <a class="reference internal" href="../../modules/naive_bayes.html#naive-bayes"><span class="std std-ref">naïve Bayes</span></a>
classifier, which
provides a nice baseline for this task. <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> includes several
variants of this classifier; the one most suitable for word counts is the
multinomial variant:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.naive_bayes</span> <span class="kn">import</span> <span class="n">MultinomialNB</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">MultinomialNB</span><span class="p">()</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train_tfidf</span><span class="p">,</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
</pre></div>
</div>
<p>To try to predict the outcome on a new document we need to extract
the features using almost the same feature extracting chain as before.
The difference is that we call <code class="docutils literal notranslate"><span class="pre">transform</span></code> instead of <code class="docutils literal notranslate"><span class="pre">fit_transform</span></code>
on the transformers, since they have already been fit to the training set:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">docs_new</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;God is love&#39;</span><span class="p">,</span> <span class="s1">&#39;OpenGL on the GPU is fast&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_new_counts</span> <span class="o">=</span> <span class="n">count_vect</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">docs_new</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_new_tfidf</span> <span class="o">=</span> <span class="n">tfidf_transformer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_new_counts</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">predicted</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_new_tfidf</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">doc</span><span class="p">,</span> <span class="n">category</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">docs_new</span><span class="p">,</span> <span class="n">predicted</span><span class="p">):</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%r</span><span class="s1"> =&gt; </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">[</span><span class="n">category</span><span class="p">]))</span>
<span class="gp">...</span>
<span class="go">&#39;God is love&#39; =&gt; soc.religion.christian</span>
<span class="go">&#39;OpenGL on the GPU is fast&#39; =&gt; comp.graphics</span>
</pre></div>
</div>
</div>
<div class="section" id="building-a-pipeline">
<h2>Building a pipeline<a class="headerlink" href="#building-a-pipeline" title="Permalink to this headline">¶</a></h2>
<p>In order to make the vectorizer =&gt; transformer =&gt; classifier easier
to work with, <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> provides a <a class="reference internal" href="../../modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline" title="sklearn.pipeline.Pipeline"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pipeline</span></code></a> class that behaves
like a compound classifier:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.pipeline</span> <span class="kn">import</span> <span class="n">Pipeline</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">text_clf</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">([</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;vect&#39;</span><span class="p">,</span> <span class="n">CountVectorizer</span><span class="p">()),</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;tfidf&#39;</span><span class="p">,</span> <span class="n">TfidfTransformer</span><span class="p">()),</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;clf&#39;</span><span class="p">,</span> <span class="n">MultinomialNB</span><span class="p">()),</span>
<span class="gp">... </span><span class="p">])</span>
</pre></div>
</div>
<p>The names <code class="docutils literal notranslate"><span class="pre">vect</span></code>, <code class="docutils literal notranslate"><span class="pre">tfidf</span></code> and <code class="docutils literal notranslate"><span class="pre">clf</span></code> (classifier) are arbitrary.
We will use them to perform grid search for suitable hyperparameters below.
We can now train the model with a single command:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">text_clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">Pipeline(...)</span>
</pre></div>
</div>
</div>
<div class="section" id="evaluation-of-the-performance-on-the-test-set">
<h2>Evaluation of the performance on the test set<a class="headerlink" href="#evaluation-of-the-performance-on-the-test-set" title="Permalink to this headline">¶</a></h2>
<p>Evaluating the predictive accuracy of the model is equally easy:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">twenty_test</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;test&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">docs_test</span> <span class="o">=</span> <span class="n">twenty_test</span><span class="o">.</span><span class="n">data</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">predicted</span> <span class="o">=</span> <span class="n">text_clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">docs_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">predicted</span> <span class="o">==</span> <span class="n">twenty_test</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">0.8348...</span>
</pre></div>
</div>
<p>We achieved 83.5% accuracy. Let’s see if we can do better with a
linear <a class="reference internal" href="../../modules/svm.html#svm"><span class="std std-ref">support vector machine (SVM)</span></a>,
which is widely regarded as one of
the best text classification algorithms (although it’s also a bit slower
than naïve Bayes). We can change the learner by simply plugging a different
classifier object into our pipeline:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">SGDClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">text_clf</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">([</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;vect&#39;</span><span class="p">,</span> <span class="n">CountVectorizer</span><span class="p">()),</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;tfidf&#39;</span><span class="p">,</span> <span class="n">TfidfTransformer</span><span class="p">()),</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s1">&#39;clf&#39;</span><span class="p">,</span> <span class="n">SGDClassifier</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s1">&#39;hinge&#39;</span><span class="p">,</span> <span class="n">penalty</span><span class="o">=</span><span class="s1">&#39;l2&#39;</span><span class="p">,</span>
<span class="gp">... </span>                          <span class="n">alpha</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">,</span>
<span class="gp">... </span>                          <span class="n">max_iter</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="kc">None</span><span class="p">)),</span>
<span class="gp">... </span><span class="p">])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">text_clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">Pipeline(...)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">predicted</span> <span class="o">=</span> <span class="n">text_clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">docs_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">predicted</span> <span class="o">==</span> <span class="n">twenty_test</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">0.9101...</span>
</pre></div>
</div>
<p>We achieved 91.3% accuracy using the SVM. <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> provides further
utilities for more detailed performance analysis of the results:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">metrics</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">metrics</span><span class="o">.</span><span class="n">classification_report</span><span class="p">(</span><span class="n">twenty_test</span><span class="o">.</span><span class="n">target</span><span class="p">,</span> <span class="n">predicted</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">target_names</span><span class="o">=</span><span class="n">twenty_test</span><span class="o">.</span><span class="n">target_names</span><span class="p">))</span>
<span class="go">                        precision    recall  f1-score   support</span>

<span class="go">           alt.atheism       0.95      0.80      0.87       319</span>
<span class="go">         comp.graphics       0.87      0.98      0.92       389</span>
<span class="go">               sci.med       0.94      0.89      0.91       396</span>
<span class="go">soc.religion.christian       0.90      0.95      0.93       398</span>

<span class="go">              accuracy                           0.91      1502</span>
<span class="go">             macro avg       0.91      0.91      0.91      1502</span>
<span class="go">          weighted avg       0.91      0.91      0.91      1502</span>


<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">confusion_matrix</span><span class="p">(</span><span class="n">twenty_test</span><span class="o">.</span><span class="n">target</span><span class="p">,</span> <span class="n">predicted</span><span class="p">)</span>
<span class="go">array([[256,  11,  16,  36],</span>
<span class="go">       [  4, 380,   3,   2],</span>
<span class="go">       [  5,  35, 353,   3],</span>
<span class="go">       [  5,  11,   4, 378]])</span>
</pre></div>
</div>
<p>As expected the confusion matrix shows that posts from the newsgroups
on atheism and Christianity are more often confused for one another than
with computer graphics.</p>
</div>
<div class="section" id="parameter-tuning-using-grid-search">
<h2>Parameter tuning using grid search<a class="headerlink" href="#parameter-tuning-using-grid-search" title="Permalink to this headline">¶</a></h2>
<p>We’ve already encountered some parameters such as <code class="docutils literal notranslate"><span class="pre">use_idf</span></code> in the
<code class="docutils literal notranslate"><span class="pre">TfidfTransformer</span></code>. Classifiers tend to have many parameters as well;
e.g., <code class="docutils literal notranslate"><span class="pre">MultinomialNB</span></code> includes a smoothing parameter <code class="docutils literal notranslate"><span class="pre">alpha</span></code> and
<code class="docutils literal notranslate"><span class="pre">SGDClassifier</span></code> has a penalty parameter <code class="docutils literal notranslate"><span class="pre">alpha</span></code> and configurable loss
and penalty terms in the objective function (see the module documentation,
or use the Python <code class="docutils literal notranslate"><span class="pre">help</span></code> function to get a description of these).</p>
<p>Instead of tweaking the parameters of the various components of the
chain, it is possible to run an exhaustive search of the best
parameters on a grid of possible values. We try out all classifiers
on either words or bigrams, with or without idf, and with a penalty
parameter of either 0.01 or 0.001 for the linear SVM:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">GridSearchCV</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">parameters</span> <span class="o">=</span> <span class="p">{</span>
<span class="gp">... </span>    <span class="s1">&#39;vect__ngram_range&#39;</span><span class="p">:</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)],</span>
<span class="gp">... </span>    <span class="s1">&#39;tfidf__use_idf&#39;</span><span class="p">:</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
<span class="gp">... </span>    <span class="s1">&#39;clf__alpha&#39;</span><span class="p">:</span> <span class="p">(</span><span class="mf">1e-2</span><span class="p">,</span> <span class="mf">1e-3</span><span class="p">),</span>
<span class="gp">... </span><span class="p">}</span>
</pre></div>
</div>
<p>Obviously, such an exhaustive search can be expensive. If we have multiple
CPU cores at our disposal, we can tell the grid searcher to try these eight
parameter combinations in parallel with the <code class="docutils literal notranslate"><span class="pre">n_jobs</span></code> parameter. If we give
this parameter a value of <code class="docutils literal notranslate"><span class="pre">-1</span></code>, grid search will detect how many cores
are installed and use them all:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">gs_clf</span> <span class="o">=</span> <span class="n">GridSearchCV</span><span class="p">(</span><span class="n">text_clf</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
</pre></div>
</div>
<p>The grid search instance behaves like a normal <code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code>
model. Let’s perform the search on a smaller subset of the training data
to speed up the computation:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">gs_clf</span> <span class="o">=</span> <span class="n">gs_clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">twenty_train</span><span class="o">.</span><span class="n">data</span><span class="p">[:</span><span class="mi">400</span><span class="p">],</span> <span class="n">twenty_train</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">400</span><span class="p">])</span>
</pre></div>
</div>
<p>The result of calling <code class="docutils literal notranslate"><span class="pre">fit</span></code> on a <code class="docutils literal notranslate"><span class="pre">GridSearchCV</span></code> object is a classifier
that we can use to <code class="docutils literal notranslate"><span class="pre">predict</span></code>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">twenty_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">[</span><span class="n">gs_clf</span><span class="o">.</span><span class="n">predict</span><span class="p">([</span><span class="s1">&#39;God is love&#39;</span><span class="p">])[</span><span class="mi">0</span><span class="p">]]</span>
<span class="go">&#39;soc.religion.christian&#39;</span>
</pre></div>
</div>
<p>The object’s <code class="docutils literal notranslate"><span class="pre">best_score_</span></code> and <code class="docutils literal notranslate"><span class="pre">best_params_</span></code> attributes store the best
mean score and the parameters setting corresponding to that score:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">gs_clf</span><span class="o">.</span><span class="n">best_score_</span>
<span class="go">0.9...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">param_name</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()):</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">param_name</span><span class="p">,</span> <span class="n">gs_clf</span><span class="o">.</span><span class="n">best_params_</span><span class="p">[</span><span class="n">param_name</span><span class="p">]))</span>
<span class="gp">...</span>
<span class="go">clf__alpha: 0.001</span>
<span class="go">tfidf__use_idf: True</span>
<span class="go">vect__ngram_range: (1, 1)</span>
</pre></div>
</div>
<p>A more detailed summary of the search is available at <code class="docutils literal notranslate"><span class="pre">gs_clf.cv_results_</span></code>.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">cv_results_</span></code> parameter can be easily imported into pandas as a
<code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> for further inspection.</p>
<div class="section" id="exercises">
<h3>Exercises<a class="headerlink" href="#exercises" title="Permalink to this headline">¶</a></h3>
<p>To do the exercises, copy the content of the ‘skeletons’ folder as
a new folder named ‘workspace’:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">%</span> <span class="n">cp</span> <span class="o">-</span><span class="n">r</span> <span class="n">skeletons</span> <span class="n">workspace</span>
</pre></div>
</div>
<p>You can then edit the content of the workspace without fear of losing
the original exercise instructions.</p>
<p>Then fire an ipython shell and run the work-in-progress script with:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">%</span><span class="n">run</span> <span class="n">workspace</span><span class="o">/</span><span class="n">exercise_XX_script</span><span class="o">.</span><span class="n">py</span> <span class="n">arg1</span> <span class="n">arg2</span> <span class="n">arg3</span>
</pre></div>
</div>
<p>If an exception is triggered, use <code class="docutils literal notranslate"><span class="pre">%debug</span></code> to fire-up a post
mortem ipdb session.</p>
<p>Refine the implementation and iterate until the exercise is solved.</p>
<p><strong>For each exercise, the skeleton file provides all the necessary import
statements, boilerplate code to load the data and sample code to evaluate
the predictive accuracy of the model.</strong></p>
</div>
</div>
<div class="section" id="exercise-1-language-identification">
<h2>Exercise 1: Language identification<a class="headerlink" href="#exercise-1-language-identification" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Write a text classification pipeline using a custom preprocessor and
<code class="docutils literal notranslate"><span class="pre">CharNGramAnalyzer</span></code> using data from Wikipedia articles as training set.</p></li>
<li><p>Evaluate the performance on some held out test set.</p></li>
</ul>
<p>ipython command line:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="n">run</span> <span class="n">workspace</span><span class="o">/</span><span class="n">exercise_01_language_train_model</span><span class="o">.</span><span class="n">py</span> <span class="n">data</span><span class="o">/</span><span class="n">languages</span><span class="o">/</span><span class="n">paragraphs</span><span class="o">/</span>
</pre></div>
</div>
</div>
<div class="section" id="exercise-2-sentiment-analysis-on-movie-reviews">
<h2>Exercise 2: Sentiment Analysis on movie reviews<a class="headerlink" href="#exercise-2-sentiment-analysis-on-movie-reviews" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Write a text classification pipeline to classify movie reviews as either
positive or negative.</p></li>
<li><p>Find a good set of parameters using grid search.</p></li>
<li><p>Evaluate the performance on a held out test set.</p></li>
</ul>
<p>ipython command line:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="n">run</span> <span class="n">workspace</span><span class="o">/</span><span class="n">exercise_02_sentiment</span><span class="o">.</span><span class="n">py</span> <span class="n">data</span><span class="o">/</span><span class="n">movie_reviews</span><span class="o">/</span><span class="n">txt_sentoken</span><span class="o">/</span>
</pre></div>
</div>
</div>
<div class="section" id="exercise-3-cli-text-classification-utility">
<h2>Exercise 3: CLI text classification utility<a class="headerlink" href="#exercise-3-cli-text-classification-utility" title="Permalink to this headline">¶</a></h2>
<p>Using the results of the previous exercises and the <code class="docutils literal notranslate"><span class="pre">cPickle</span></code>
module of the standard library, write a command line utility that
detects the language of some text provided on <code class="docutils literal notranslate"><span class="pre">stdin</span></code> and estimate
the polarity (positive or negative) if the text is written in
English.</p>
<p>Bonus point if the utility is able to give a confidence level for its
predictions.</p>
</div>
<div class="section" id="where-to-from-here">
<h2>Where to from here<a class="headerlink" href="#where-to-from-here" title="Permalink to this headline">¶</a></h2>
<p>Here are a few suggestions to help further your scikit-learn intuition
upon the completion of this tutorial:</p>
<ul class="simple">
<li><p>Try playing around with the <code class="docutils literal notranslate"><span class="pre">analyzer</span></code> and <code class="docutils literal notranslate"><span class="pre">token</span> <span class="pre">normalisation</span></code> under
<a class="reference internal" href="../../modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer" title="sklearn.feature_extraction.text.CountVectorizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">CountVectorizer</span></code></a>.</p></li>
<li><p>If you don’t have labels, try using
<a class="reference internal" href="../../auto_examples/text/plot_document_clustering.html#sphx-glr-auto-examples-text-plot-document-clustering-py"><span class="std std-ref">Clustering</span></a>
on your problem.</p></li>
<li><p>If you have multiple labels per document, e.g categories, have a look
at the <a class="reference internal" href="../../modules/multiclass.html#multiclass"><span class="std std-ref">Multiclass and multilabel section</span></a>.</p></li>
<li><p>Try using <a class="reference internal" href="../../modules/decomposition.html#lsa"><span class="std std-ref">Truncated SVD</span></a> for
<a class="reference external" href="https://en.wikipedia.org/wiki/Latent_semantic_analysis">latent semantic analysis</a>.</p></li>
<li><p>Have a look at using
<a class="reference internal" href="../../auto_examples/applications/plot_out_of_core_classification.html#sphx-glr-auto-examples-applications-plot-out-of-core-classification-py"><span class="std std-ref">Out-of-core Classification</span></a> to
learn from data that would not fit into the computer main memory.</p></li>
<li><p>Have a look at the <a class="reference internal" href="../../modules/feature_extraction.html#hashing-vectorizer"><span class="std std-ref">Hashing Vectorizer</span></a>
as a memory efficient alternative to <a class="reference internal" href="../../modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer" title="sklearn.feature_extraction.text.CountVectorizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">CountVectorizer</span></code></a>.</p></li>
</ul>
</div>
</div>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2019, scikit-learn developers (BSD License).
          <a href="../../_sources/tutorial/text_analytics/working_with_text_data.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');
    var hide_text = 'Hide prompts and outputs';
    var show_text = 'Show prompts and outputs';

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        if (jthis.find('.gp').length > 0) {
            var button = $('<span class="copybutton">&gt;&gt;&gt;</span>');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
            jthis.prepend(button);
        }
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

    // define the behavior of the button when it's clicked
    $('.copybutton').click(function(e){
        e.preventDefault();
        var button = $(this);
        if (button.data('hidden') === 'false') {
            // hide the code output
            button.parent().find('.go, .gp, .gt').hide();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
            button.css('text-decoration', 'line-through');
            button.attr('title', show_text);
            button.data('hidden', 'true');
        } else {
            // show the code output
            button.parent().find('.go, .gp, .gt').show();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
            button.css('text-decoration', 'none');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
        }
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
  /*** Hide navbar when scrolling down ***/
  // Returns true when headerlink target matches hash in url
  (function() {
    hashTargetOnTop = function() {
        var hash = window.location.hash;
        if ( hash.length < 2 ) { return false; }

        var target = document.getElementById( hash.slice(1) );
        if ( target === null ) { return false; }

        var top = target.getBoundingClientRect().top;
        return (top < 2) && (top > -2);
    };

    // Hide navbar on load if hash target is on top
    var navBar = document.getElementById("navbar");
    var navBarToggler = document.getElementById("sk-navbar-toggler");
    var navBarHeightHidden = "-" + navBar.getBoundingClientRect().height + "px";
    var $window = $(window);

    hideNavBar = function() {
        navBar.style.top = navBarHeightHidden;
    };

    showNavBar = function() {
        navBar.style.top = "0";
    }

    if (hashTargetOnTop()) {
        hideNavBar()
    }

    var prevScrollpos = window.pageYOffset;
    hideOnScroll = function(lastScrollTop) {
        if (($window.width() < 768) && (navBarToggler.getAttribute("aria-expanded") === 'true')) {
            return;
        }
        if (lastScrollTop > 2 && (prevScrollpos <= lastScrollTop) || hashTargetOnTop()){
            hideNavBar()
        } else {
            showNavBar()
        }
        prevScrollpos = lastScrollTop;
    };

    /*** high preformance scroll event listener***/
    var raf = window.requestAnimationFrame ||
        window.webkitRequestAnimationFrame ||
        window.mozRequestAnimationFrame ||
        window.msRequestAnimationFrame ||
        window.oRequestAnimationFrame;
    var lastScrollTop = $window.scrollTop();

    if (raf) {
        loop();
    }

    function loop() {
        var scrollTop = $window.scrollTop();
        if (lastScrollTop === scrollTop) {
            raf(loop);
            return;
        } else {
            lastScrollTop = scrollTop;
            hideOnScroll(lastScrollTop);
            raf(loop);
        }
    }
  })();
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
</body>
</html>